{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JCG15IOGm-cY"
      },
      "source": [
        "##### Copyright 2020 Google LLC.\n",
        "\n",
        "\n",
        "Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "you may not use this file except in compliance with the License.\n",
        "You may obtain a copy of the License at\n",
        "\n",
        "    https://www.apache.org/licenses/LICENSE-2.0\n",
        "\n",
        "Unless required by applicable law or agreed to in writing, software\n",
        "distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "See the License for the specific language governing permissions and\n",
        "limitations under the License."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "executionInfo": {
          "elapsed": 2819,
          "status": "ok",
          "timestamp": 1593126151058,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "6n1hColO-89V",
        "outputId": "df661a62-ad35-42e5-9d0b-2f5cee6bde04"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: python-Levenshtein in /usr/local/lib/python3.6/dist-packages (0.12.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from python-Levenshtein) (47.3.1)\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "import numpy\n",
        "import pandas\n",
        "from six.moves import zip\n",
        "from sklearn import mixture\n",
        "import gzip\n",
        "\n",
        "!pip install python-Levenshtein\n",
        "\n",
        "import Levenshtein"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vnv4BxyUsMz7"
      },
      "source": [
        "## Code to fit GMM"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KXnqP29_sAXj"
      },
      "outputs": [],
      "source": [
        "R1_TILE21_WT_SEQ = 'DEEEIRTTNPVATEQYGSVSTNLQRGNR'\n",
        "\n",
        "# Covariance type to use in Gaussian Mixture Model.\n",
        "_COVAR_TYPE = 'full'\n",
        "# Number of components to use in Gaussian Mixture Model.\n",
        "_NUM_COMPONENTS = 2\n",
        "\n",
        "class BinningLabeler(object):\n",
        "  \"\"\"Emits class labels from provided cutoff values.\n",
        "\n",
        "  Input cutoffs are encoded as 1-D arrays.  Given a cutoffs array of\n",
        "  size n, creates n+1 labels for cutoffs, where the first bin is\n",
        "  [-inf, cutoffs[0]], and last bin is (cutoffs[-1], inf].\n",
        "  \"\"\"\n",
        "\n",
        "  def __init__(self, cutoffs):\n",
        "    \"\"\"Constructor.\n",
        "\n",
        "    Args:\n",
        "      cutoffs: (numpy.ndarray or list or numeric) values to bin data at.  First bin\n",
        "        is [-inf, cutoffs[0]], and last bin is (cutoffs[-1], inf].\n",
        "\n",
        "    Raises:\n",
        "      ValueError: If no cutoff(s) (i.e. an empty list) is provided.\n",
        "    \"\"\"\n",
        "    cutoffs = numpy.atleast_1d(cutoffs)\n",
        "    if cutoffs.size:\n",
        "      self._cutoffs = numpy.sort(cutoffs)\n",
        "    else:\n",
        "      raise ValueError('Invalid cutoffs. At least one cutoff value required.')\n",
        "\n",
        "  def predict(self, values):\n",
        "    \"\"\"Provides model labels for input value(s) using the cutoff bins.\n",
        "\n",
        "    Args:\n",
        "      values: (numpy.ndarray or numeric) Value(s) to infer a label on.\n",
        "\n",
        "    Returns:\n",
        "      A numpy array with length len(values) and labels corresponding to\n",
        "      categories defined by the cutoffs array intervals.  The labels are\n",
        "      [0, 1, . . ., n], where n = len(cutoffs).  Note, labels correspond to bins\n",
        "      in sorted order from smallest to largest cutoff value.\n",
        "    \"\"\"\n",
        "    return numpy.digitize(values, self._cutoffs)\n",
        "\n",
        "\n",
        "class TwoGaussianMixtureModelLabeler(object):\n",
        "  \"\"\"Emits class labels from Gaussian Mixture given input data.\n",
        "\n",
        "  Input data is encoded as 1-D arrays.  Allows for an optional ambiguous label\n",
        "  between the two modelled Gaussian distributions. Without the optional\n",
        "  ambigouous category, the two labels are:\n",
        "     0 - For values more likely derived from the Gaussian with smaller mean\n",
        "     2 - For values more likely derived from the Gaussian with larger mean\n",
        "\n",
        "  When allowing for an ambiguous category the three labels are:\n",
        "     0 - For values more likely derived from the Gaussian with smaller mean\n",
        "     1 - For values which fall within an ambiguous probability cutoff.\n",
        "     2 - For values more likely derived from the Gaussian with larger mean\n",
        "  \"\"\"\n",
        "\n",
        "  def __init__(self, data):\n",
        "    \"\"\"Constructor.\n",
        "\n",
        "    Args:\n",
        "      data: (numpy.ndarray or list) Input data to model with Gaussian Mixture.\n",
        "      Input data is presumed to be in the form [x1, x2, ...., xn].\n",
        "    \"\"\"\n",
        "    self._data = numpy.array([data]).T\n",
        "    self._gmm = mixture.GaussianMixture(\n",
        "        n_components=_NUM_COMPONENTS,\n",
        "        covariance_type=_COVAR_TYPE).fit(self._data)\n",
        "\n",
        "    # Re-map the gaussian with smaller mean to the \"0\" label.\n",
        "    self._label_by_index = dict(\n",
        "        list(zip([0, 1],\n",
        "                 numpy.argsort(self._gmm.means_[:, 0]).tolist())))\n",
        "    self._label_by_index_fn = numpy.vectorize(lambda x: self._label_by_index[x])\n",
        "\n",
        "  def predict(self, values, probability_cutoff=0.):\n",
        "    \"\"\"Provides model labels for input value(s) using the GMM.\n",
        "\n",
        "    Args:\n",
        "      values: (array or single float value) Value(s) to infer a label on.\n",
        "        When values=None, predictions are run on self._data.\n",
        "      probability_cutoff: (float) Proability between 0 and 1 to identify which\n",
        "        values correspond to ambiguous labels.  At probablity_cutoff=0 (default)\n",
        "        it only returns the original two state predictions.\n",
        "\n",
        "    Returns:\n",
        "      A numpy array with length len(values) and labels corresponding to 0,1 if\n",
        "      probability_cutoff = 0 and 0, 1, 2 otherwise.  In the latter, 0\n",
        "      corresponds to the gaussian with smaller mean, 1 corresponds to the\n",
        "      ambiguous label, and 2 corresponds to the gaussian with larger mean.\n",
        "    \"\"\"\n",
        "    values = numpy.atleast_1d(values)\n",
        "    values = numpy.array([values]).T\n",
        "    predictions = self._label_by_index_fn(self._gmm.predict(values))\n",
        "    # Re-map the initial 0,1 predictions to 0,2.\n",
        "    predictions *= 2\n",
        "    if probability_cutoff \u003e 0:\n",
        "      probas = self._gmm.predict_proba(values)\n",
        "      max_probas = numpy.max(probas, axis=1)\n",
        "      ambiguous_values = max_probas \u003c probability_cutoff\n",
        "\n",
        "      # Set ambiguous label as 1.\n",
        "      predictions[ambiguous_values] = 1\n",
        "    return predictions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wqRjTF3mWJK7"
      },
      "source": [
        "## Load validation experiment dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 232
        },
        "executionInfo": {
          "elapsed": 533,
          "status": "ok",
          "timestamp": 1593126523588,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "am0nFjW9_F25",
        "outputId": "840fda97-2a86-4ced-d1f2-ed6d9e3d020a"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\u003cdiv\u003e\n",
              "\u003cstyle scoped\u003e\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "\u003c/style\u003e\n",
              "\u003ctable border=\"1\" class=\"dataframe\"\u003e\n",
              "  \u003cthead\u003e\n",
              "    \u003ctr style=\"text-align: right;\"\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003eUnnamed: 0\u003c/th\u003e\n",
              "      \u003cth\u003eUnnamed: 0.1\u003c/th\u003e\n",
              "      \u003cth\u003esequence\u003c/th\u003e\n",
              "      \u003cth\u003epartition\u003c/th\u003e\n",
              "      \u003cth\u003echip\u003c/th\u003e\n",
              "      \u003cth\u003econtrol\u003c/th\u003e\n",
              "      \u003cth\u003eis_wt_aa\u003c/th\u003e\n",
              "      \u003cth\u003eis_wt_nt\u003c/th\u003e\n",
              "      \u003cth\u003emutation_sequence\u003c/th\u003e\n",
              "      \u003cth\u003enum_mutations\u003c/th\u003e\n",
              "      \u003cth\u003erep_i\u003c/th\u003e\n",
              "      \u003cth\u003erep_original\u003c/th\u003e\n",
              "      \u003cth\u003erep_total\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1a_plasmid_x\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1b_plasmid_x\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1c_plasmid_x\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1d_plasmid_x\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v3_rep1a_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v3_rep1b_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v3_rep1c_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v3_rep1d_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v4_rep2a_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v4_rep2b_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v4_rep2c_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v4_rep2d_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v5_rep3a_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v5_rep3b_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1a_plasmid_y\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1b_plasmid_y\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1c_plasmid_y\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1d_plasmid_y\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v3_rep1a_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v3_rep1b_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v3_rep1c_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v3_rep1d_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v4_rep2a_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v4_rep2b_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v4_rep2c_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v4_rep2d_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v5_rep3a_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v5_rep3b_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v5_rep3c_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v5_rep3d_virus\u003c/th\u003e\n",
              "      \u003cth\u003eGAS1_plasmid_N\u003c/th\u003e\n",
              "      \u003cth\u003ev0_GAS1_plasmid_N\u003c/th\u003e\n",
              "      \u003cth\u003ev1_GAS1_plasmid_N\u003c/th\u003e\n",
              "      \u003cth\u003ev0_GAS1_virus_N\u003c/th\u003e\n",
              "      \u003cth\u003ev1_GAS1_virus_N\u003c/th\u003e\n",
              "      \u003cth\u003eGAS1_virus_N\u003c/th\u003e\n",
              "      \u003cth\u003ev0_GAS1_plasmid_F\u003c/th\u003e\n",
              "      \u003cth\u003ev1_GAS1_plasmid_F\u003c/th\u003e\n",
              "      \u003cth\u003eGAS1_plasmid_F\u003c/th\u003e\n",
              "      \u003cth\u003ev0_GAS1_virus_F\u003c/th\u003e\n",
              "      \u003cth\u003ev1_GAS1_virus_F\u003c/th\u003e\n",
              "      \u003cth\u003eGAS1_virus_F\u003c/th\u003e\n",
              "      \u003cth\u003ev0_GAS1_virus_S\u003c/th\u003e\n",
              "      \u003cth\u003ev1_GAS1_virus_S\u003c/th\u003e\n",
              "      \u003cth\u003eGAS1_virus_S\u003c/th\u003e\n",
              "    \u003c/tr\u003e\n",
              "  \u003c/thead\u003e\n",
              "  \u003ctbody\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e0\u003c/th\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003eGACGAGGACGAAATCAGGACAACCAATCCCGTGGCTACGGAGCAGT...\u003c/td\u003e\n",
              "      \u003ctd\u003eDEDEIRTTNPVATEQYGSVSTNLQDnGnNdR\u003c/td\u003e\n",
              "      \u003ctd\u003ernn_designed_plus_rand_train_walked\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e__D_____________________Dn_n_d_\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e13\u003c/td\u003e\n",
              "      \u003ctd\u003e89\u003c/td\u003e\n",
              "      \u003ctd\u003e51\u003c/td\u003e\n",
              "      \u003ctd\u003e47\u003c/td\u003e\n",
              "      \u003ctd\u003e176\u003c/td\u003e\n",
              "      \u003ctd\u003e47\u003c/td\u003e\n",
              "      \u003ctd\u003e53\u003c/td\u003e\n",
              "      \u003ctd\u003e211\u003c/td\u003e\n",
              "      \u003ctd\u003e78\u003c/td\u003e\n",
              "      \u003ctd\u003e188\u003c/td\u003e\n",
              "      \u003ctd\u003e84\u003c/td\u003e\n",
              "      \u003ctd\u003e29\u003c/td\u003e\n",
              "      \u003ctd\u003e45\u003c/td\u003e\n",
              "      \u003ctd\u003e42\u003c/td\u003e\n",
              "      \u003ctd\u003e6.0\u003c/td\u003e\n",
              "      \u003ctd\u003e8.0\u003c/td\u003e\n",
              "      \u003ctd\u003e8.0\u003c/td\u003e\n",
              "      \u003ctd\u003e8.0\u003c/td\u003e\n",
              "      \u003ctd\u003e16.0\u003c/td\u003e\n",
              "      \u003ctd\u003e17.0\u003c/td\u003e\n",
              "      \u003ctd\u003e16.0\u003c/td\u003e\n",
              "      \u003ctd\u003e12.0\u003c/td\u003e\n",
              "      \u003ctd\u003e17.0\u003c/td\u003e\n",
              "      \u003ctd\u003e18.0\u003c/td\u003e\n",
              "      \u003ctd\u003e19.0\u003c/td\u003e\n",
              "      \u003ctd\u003e8.0\u003c/td\u003e\n",
              "      \u003ctd\u003e21.0\u003c/td\u003e\n",
              "      \u003ctd\u003e16.0\u003c/td\u003e\n",
              "      \u003ctd\u003e10.0\u003c/td\u003e\n",
              "      \u003ctd\u003e20.0\u003c/td\u003e\n",
              "      \u003ctd\u003e230.0\u003c/td\u003e\n",
              "      \u003ctd\u003e30.0\u003c/td\u003e\n",
              "      \u003ctd\u003e200\u003c/td\u003e\n",
              "      \u003ctd\u003e190.0\u003c/td\u003e\n",
              "      \u003ctd\u003e953\u003c/td\u003e\n",
              "      \u003ctd\u003e1143.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000004\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000003\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000003\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000006\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000005\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000005\u003c/td\u003e\n",
              "      \u003ctd\u003e0.579811\u003c/td\u003e\n",
              "      \u003ctd\u003e0.561301\u003c/td\u003e\n",
              "      \u003ctd\u003e0.572274\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e1\u003c/th\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003eGACGAGGACGAAATCAGGACAACCAATCCCGTGGCTACGGAGCAGT...\u003c/td\u003e\n",
              "      \u003ctd\u003eDEDEIRTTNPVATEQYGAVSTNLQGdGNdR\u003c/td\u003e\n",
              "      \u003ctd\u003ernn_designed_plus_rand_train_walked\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e__D______________A______Gd__d_\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e29\u003c/td\u003e\n",
              "      \u003ctd\u003e129\u003c/td\u003e\n",
              "      \u003ctd\u003e100\u003c/td\u003e\n",
              "      \u003ctd\u003e63\u003c/td\u003e\n",
              "      \u003ctd\u003e439\u003c/td\u003e\n",
              "      \u003ctd\u003e131\u003c/td\u003e\n",
              "      \u003ctd\u003e130\u003c/td\u003e\n",
              "      \u003ctd\u003e563\u003c/td\u003e\n",
              "      \u003ctd\u003e269\u003c/td\u003e\n",
              "      \u003ctd\u003e513\u003c/td\u003e\n",
              "      \u003ctd\u003e270\u003c/td\u003e\n",
              "      \u003ctd\u003e98\u003c/td\u003e\n",
              "      \u003ctd\u003e88\u003c/td\u003e\n",
              "      \u003ctd\u003e85\u003c/td\u003e\n",
              "      \u003ctd\u003e11.0\u003c/td\u003e\n",
              "      \u003ctd\u003e12.0\u003c/td\u003e\n",
              "      \u003ctd\u003e13.0\u003c/td\u003e\n",
              "      \u003ctd\u003e13.0\u003c/td\u003e\n",
              "      \u003ctd\u003e36.0\u003c/td\u003e\n",
              "      \u003ctd\u003e47.0\u003c/td\u003e\n",
              "      \u003ctd\u003e34.0\u003c/td\u003e\n",
              "      \u003ctd\u003e50.0\u003c/td\u003e\n",
              "      \u003ctd\u003e41.0\u003c/td\u003e\n",
              "      \u003ctd\u003e51.0\u003c/td\u003e\n",
              "      \u003ctd\u003e25.0\u003c/td\u003e\n",
              "      \u003ctd\u003e20.0\u003c/td\u003e\n",
              "      \u003ctd\u003e27.0\u003c/td\u003e\n",
              "      \u003ctd\u003e36.0\u003c/td\u003e\n",
              "      \u003ctd\u003e42.0\u003c/td\u003e\n",
              "      \u003ctd\u003e47.0\u003c/td\u003e\n",
              "      \u003ctd\u003e370.0\u003c/td\u003e\n",
              "      \u003ctd\u003e49.0\u003c/td\u003e\n",
              "      \u003ctd\u003e321\u003c/td\u003e\n",
              "      \u003ctd\u003e456.0\u003c/td\u003e\n",
              "      \u003ctd\u003e2586\u003c/td\u003e\n",
              "      \u003ctd\u003e3042.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000007\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000005\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000006\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000014\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000014\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000014\u003c/td\u003e\n",
              "      \u003ctd\u003e1.135026\u003c/td\u003e\n",
              "      \u003ctd\u003e1.318902\u003c/td\u003e\n",
              "      \u003ctd\u003e1.298578\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e2\u003c/th\u003e\n",
              "      \u003ctd\u003e2\u003c/td\u003e\n",
              "      \u003ctd\u003eGACGAAGAGGAAATCGCTACAACCAATCCCGTGGCTACGGAGCAGT...\u003c/td\u003e\n",
              "      \u003ctd\u003eDEEEIATTNPVATEQYGSVSTNLQHdGDeR\u003c/td\u003e\n",
              "      \u003ctd\u003ernn_designed_plus_rand_train_walked\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e_____A__________________Hd_De_\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e27\u003c/td\u003e\n",
              "      \u003ctd\u003e121\u003c/td\u003e\n",
              "      \u003ctd\u003e91\u003c/td\u003e\n",
              "      \u003ctd\u003e65\u003c/td\u003e\n",
              "      \u003ctd\u003e171\u003c/td\u003e\n",
              "      \u003ctd\u003e66\u003c/td\u003e\n",
              "      \u003ctd\u003e50\u003c/td\u003e\n",
              "      \u003ctd\u003e176\u003c/td\u003e\n",
              "      \u003ctd\u003e356\u003c/td\u003e\n",
              "      \u003ctd\u003e648\u003c/td\u003e\n",
              "      \u003ctd\u003e338\u003c/td\u003e\n",
              "      \u003ctd\u003e128\u003c/td\u003e\n",
              "      \u003ctd\u003e46\u003c/td\u003e\n",
              "      \u003ctd\u003e42\u003c/td\u003e\n",
              "      \u003ctd\u003e12.0\u003c/td\u003e\n",
              "      \u003ctd\u003e9.0\u003c/td\u003e\n",
              "      \u003ctd\u003e14.0\u003c/td\u003e\n",
              "      \u003ctd\u003e11.0\u003c/td\u003e\n",
              "      \u003ctd\u003e14.0\u003c/td\u003e\n",
              "      \u003ctd\u003e12.0\u003c/td\u003e\n",
              "      \u003ctd\u003e8.0\u003c/td\u003e\n",
              "      \u003ctd\u003e12.0\u003c/td\u003e\n",
              "      \u003ctd\u003e44.0\u003c/td\u003e\n",
              "      \u003ctd\u003e53.0\u003c/td\u003e\n",
              "      \u003ctd\u003e36.0\u003c/td\u003e\n",
              "      \u003ctd\u003e42.0\u003c/td\u003e\n",
              "      \u003ctd\u003e13.0\u003c/td\u003e\n",
              "      \u003ctd\u003e14.0\u003c/td\u003e\n",
              "      \u003ctd\u003e18.0\u003c/td\u003e\n",
              "      \u003ctd\u003e20.0\u003c/td\u003e\n",
              "      \u003ctd\u003e350.0\u003c/td\u003e\n",
              "      \u003ctd\u003e46.0\u003c/td\u003e\n",
              "      \u003ctd\u003e304\u003c/td\u003e\n",
              "      \u003ctd\u003e286.0\u003c/td\u003e\n",
              "      \u003ctd\u003e2021\u003c/td\u003e\n",
              "      \u003ctd\u003e2307.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000006\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000005\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000005\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000009\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000011\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000010\u003c/td\u003e\n",
              "      \u003ctd\u003e0.553155\u003c/td\u003e\n",
              "      \u003ctd\u003e1.041751\u003c/td\u003e\n",
              "      \u003ctd\u003e0.979746\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e3\u003c/th\u003e\n",
              "      \u003ctd\u003e3\u003c/td\u003e\n",
              "      \u003ctd\u003eGACGAACACGAAATCAGGACAACCAATCCCGTGGCTACGGAGCAGT...\u003c/td\u003e\n",
              "      \u003ctd\u003eDEHEIRTTNPVATEQYGNVSTNLQGgGdNR\u003c/td\u003e\n",
              "      \u003ctd\u003ernn_designed_plus_rand_train_walked\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e__H______________N______Gg_d__\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e66\u003c/td\u003e\n",
              "      \u003ctd\u003e288\u003c/td\u003e\n",
              "      \u003ctd\u003e243\u003c/td\u003e\n",
              "      \u003ctd\u003e190\u003c/td\u003e\n",
              "      \u003ctd\u003e830\u003c/td\u003e\n",
              "      \u003ctd\u003e244\u003c/td\u003e\n",
              "      \u003ctd\u003e238\u003c/td\u003e\n",
              "      \u003ctd\u003e979\u003c/td\u003e\n",
              "      \u003ctd\u003e651\u003c/td\u003e\n",
              "      \u003ctd\u003e1158\u003c/td\u003e\n",
              "      \u003ctd\u003e588\u003c/td\u003e\n",
              "      \u003ctd\u003e185\u003c/td\u003e\n",
              "      \u003ctd\u003e241\u003c/td\u003e\n",
              "      \u003ctd\u003e229\u003c/td\u003e\n",
              "      \u003ctd\u003e12.0\u003c/td\u003e\n",
              "      \u003ctd\u003e11.0\u003c/td\u003e\n",
              "      \u003ctd\u003e37.0\u003c/td\u003e\n",
              "      \u003ctd\u003e32.0\u003c/td\u003e\n",
              "      \u003ctd\u003e76.0\u003c/td\u003e\n",
              "      \u003ctd\u003e64.0\u003c/td\u003e\n",
              "      \u003ctd\u003e63.0\u003c/td\u003e\n",
              "      \u003ctd\u003e50.0\u003c/td\u003e\n",
              "      \u003ctd\u003e88.0\u003c/td\u003e\n",
              "      \u003ctd\u003e89.0\u003c/td\u003e\n",
              "      \u003ctd\u003e86.0\u003c/td\u003e\n",
              "      \u003ctd\u003e52.0\u003c/td\u003e\n",
              "      \u003ctd\u003e73.0\u003c/td\u003e\n",
              "      \u003ctd\u003e81.0\u003c/td\u003e\n",
              "      \u003ctd\u003e87.0\u003c/td\u003e\n",
              "      \u003ctd\u003e91.0\u003c/td\u003e\n",
              "      \u003ctd\u003e879.0\u003c/td\u003e\n",
              "      \u003ctd\u003e92.0\u003c/td\u003e\n",
              "      \u003ctd\u003e787\u003c/td\u003e\n",
              "      \u003ctd\u003e900.0\u003c/td\u003e\n",
              "      \u003ctd\u003e5343\u003c/td\u003e\n",
              "      \u003ctd\u003e6243.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000012\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000013\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000013\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000029\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000028\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000028\u003c/td\u003e\n",
              "      \u003ctd\u003e1.207065\u003c/td\u003e\n",
              "      \u003ctd\u003e1.072039\u003c/td\u003e\n",
              "      \u003ctd\u003e1.087459\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e4\u003c/th\u003e\n",
              "      \u003ctd\u003e4\u003c/td\u003e\n",
              "      \u003ctd\u003eGACGAACATGAAATCAGGACAACCAATCCCGTGGCTACGGAGCAGT...\u003c/td\u003e\n",
              "      \u003ctd\u003eDEHEIRTTNPVATEQYGSVSTNLQpGGNDg\u003c/td\u003e\n",
              "      \u003ctd\u003ernn_designed_plus_rand_train_walked\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e__H_____________________pG__Dg\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e36\u003c/td\u003e\n",
              "      \u003ctd\u003e172\u003c/td\u003e\n",
              "      \u003ctd\u003e122\u003c/td\u003e\n",
              "      \u003ctd\u003e97\u003c/td\u003e\n",
              "      \u003ctd\u003e552\u003c/td\u003e\n",
              "      \u003ctd\u003e188\u003c/td\u003e\n",
              "      \u003ctd\u003e157\u003c/td\u003e\n",
              "      \u003ctd\u003e616\u003c/td\u003e\n",
              "      \u003ctd\u003e487\u003c/td\u003e\n",
              "      \u003ctd\u003e845\u003c/td\u003e\n",
              "      \u003ctd\u003e457\u003c/td\u003e\n",
              "      \u003ctd\u003e146\u003c/td\u003e\n",
              "      \u003ctd\u003e75\u003c/td\u003e\n",
              "      \u003ctd\u003e76\u003c/td\u003e\n",
              "      \u003ctd\u003e15.0\u003c/td\u003e\n",
              "      \u003ctd\u003e17.0\u003c/td\u003e\n",
              "      \u003ctd\u003e10.0\u003c/td\u003e\n",
              "      \u003ctd\u003e10.0\u003c/td\u003e\n",
              "      \u003ctd\u003e50.0\u003c/td\u003e\n",
              "      \u003ctd\u003e59.0\u003c/td\u003e\n",
              "      \u003ctd\u003e22.0\u003c/td\u003e\n",
              "      \u003ctd\u003e29.0\u003c/td\u003e\n",
              "      \u003ctd\u003e63.0\u003c/td\u003e\n",
              "      \u003ctd\u003e65.0\u003c/td\u003e\n",
              "      \u003ctd\u003e77.0\u003c/td\u003e\n",
              "      \u003ctd\u003e57.0\u003c/td\u003e\n",
              "      \u003ctd\u003e18.0\u003c/td\u003e\n",
              "      \u003ctd\u003e26.0\u003c/td\u003e\n",
              "      \u003ctd\u003e37.0\u003c/td\u003e\n",
              "      \u003ctd\u003e24.0\u003c/td\u003e\n",
              "      \u003ctd\u003e479.0\u003c/td\u003e\n",
              "      \u003ctd\u003e52.0\u003c/td\u003e\n",
              "      \u003ctd\u003e427\u003c/td\u003e\n",
              "      \u003ctd\u003e527.0\u003c/td\u003e\n",
              "      \u003ctd\u003e3599\u003c/td\u003e\n",
              "      \u003ctd\u003e4126.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000007\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000007\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000007\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000017\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000019\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000019\u003c/td\u003e\n",
              "      \u003ctd\u003e1.258065\u003c/td\u003e\n",
              "      \u003ctd\u003e1.384113\u003c/td\u003e\n",
              "      \u003ctd\u003e1.365801\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "  \u003c/tbody\u003e\n",
              "\u003c/table\u003e\n",
              "\u003c/div\u003e"
            ],
            "text/plain": [
              "   Unnamed: 0  ... GAS1_virus_S\n",
              "0           0  ...     0.572274\n",
              "1           1  ...     1.298578\n",
              "2           2  ...     0.979746\n",
              "3           3  ...     1.087459\n",
              "4           4  ...     1.365801\n",
              "\n",
              "[5 rows x 58 columns]"
            ]
          },
          "execution_count": 11,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "with gzip.open('GAS1_target_20190516.csv.gz', 'rb') as f:\n",
        "    gas1 = pandas.read_csv(f, index_col=None)\n",
        "\n",
        "gas1 = gas1.rename({\n",
        "    'aa': 'sequence',\n",
        "    'mask': 'mutation_sequence',\n",
        "    'mut': 'num_mutations',\n",
        "    'category': 'partition',\n",
        "}, axis=1)\n",
        "\n",
        "gas1_orig = gas1.copy() ## for comparison below if needed\n",
        "\n",
        "gas1.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dOY5xHMsAJy3"
      },
      "source": [
        "#### Validate that N-\u003eF columns computed as expected"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EplSrzWA_V_0"
      },
      "outputs": [],
      "source": [
        "numpy.testing.assert_allclose(\n",
        "    gas1.GAS1_plasmid_F, \n",
        "    gas1.GAS1_plasmid_N / gas1.GAS1_plasmid_N.sum())\n",
        "\n",
        "numpy.testing.assert_allclose(\n",
        "    gas1.GAS1_virus_F,\n",
        "    gas1.GAS1_virus_N / gas1.GAS1_virus_N.sum())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cQM1yfpZBKoL"
      },
      "source": [
        "### Filter sequences with insufficient plasmids"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ID0Uzv53AOzU"
      },
      "source": [
        "#### Find zero-plasmid sequences"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "executionInfo": {
          "elapsed": 337,
          "status": "ok",
          "timestamp": 1593126569792,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "hVLg7uAJAbmc",
        "outputId": "86f7acb7-9037-4d91-8ad8-d54d270710d4"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "446"
            ]
          },
          "execution_count": 13,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "zero_plasmids_mask = gas1.GAS1_plasmid_N == 0\n",
        "zero_plasmids_mask.sum()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hiUGQzH_AnOM"
      },
      "source": [
        "#### Find low-plasmid count sequences\n",
        "These selection values are unreliable, more noisy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "executionInfo": {
          "elapsed": 340,
          "status": "ok",
          "timestamp": 1593126571756,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "8kB9qFH7Ats9",
        "outputId": "fee7db53-36a8-4594-9d70-2e736eb05c22"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "1887"
            ]
          },
          "execution_count": 14,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "low_plasmids_mask = (gas1.GAS1_plasmid_N \u003c 10) \u0026 ~zero_plasmids_mask\n",
        "low_plasmids_mask.sum()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TGkyGMlLBBGE"
      },
      "source": [
        "#### Drop sequences that don't meet the plasmid count bars"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "executionInfo": {
          "elapsed": 348,
          "status": "ok",
          "timestamp": 1593126572873,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "iCXmHqKAA8Sd",
        "outputId": "ab91b828-0519-4218-ad6f-84bf98ff0b7c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "2333"
            ]
          },
          "execution_count": 15,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "seqs_to_remove = (low_plasmids_mask | zero_plasmids_mask)\n",
        "seqs_to_remove.sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "executionInfo": {
          "elapsed": 328,
          "status": "ok",
          "timestamp": 1593126573929,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "krsWbYNaBVv9",
        "outputId": "f510f9a6-d17c-4159-be5a-697f41a5a020"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "243481"
            ]
          },
          "execution_count": 16,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "num_seqs_before_plasmid_filter = len(gas1)\n",
        "num_seqs_before_plasmid_filter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E3r35Jh8BGQ_"
      },
      "outputs": [],
      "source": [
        "gas1 = gas1[~seqs_to_remove].copy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "executionInfo": {
          "elapsed": 318,
          "status": "ok",
          "timestamp": 1593126577014,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "krMEp-FYBoj9",
        "outputId": "d146319d-e535-4e99-ac2e-46fcf22e2e1a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "2333"
            ]
          },
          "execution_count": 18,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "num_seqs_before_plasmid_filter - len(gas1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "executionInfo": {
          "elapsed": 317,
          "status": "ok",
          "timestamp": 1593126578145,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "wpazOmBdKx4Y",
        "outputId": "78788390-55a8-47f0-fe65-6b4b324c95c2"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "241148"
            ]
          },
          "execution_count": 19,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(gas1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uPI3xpmAwPvV"
      },
      "source": [
        "###  Add pseudocounts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7eZarz4gwPgf"
      },
      "outputs": [],
      "source": [
        "PSEUDOCOUNT = 1\n",
        "\n",
        "def counts_to_frequency(counts):\n",
        "  return counts / counts.sum()\n",
        "\n",
        "\n",
        "gas1['virus_N'] = gas1.GAS1_virus_N + PSEUDOCOUNT\n",
        "gas1['plasmid_N'] = gas1.GAS1_plasmid_N + PSEUDOCOUNT\n",
        "\n",
        "gas1['virus_F'] = counts_to_frequency(gas1.virus_N)\n",
        "gas1['plasmid_F'] = counts_to_frequency(gas1.plasmid_N)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GedLhfwzCngt"
      },
      "source": [
        "### Compute viral selection "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        },
        "executionInfo": {
          "elapsed": 308,
          "status": "ok",
          "timestamp": 1593126582996,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "khDxPsIWzhM7",
        "outputId": "de52006c-a480-44b7-ec55-af0b35e4a59f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "count    241148.000000\n",
              "mean         -2.601064\n",
              "std           3.393798\n",
              "min         -11.176109\n",
              "25%          -5.814133\n",
              "50%          -2.479348\n",
              "75%           0.761311\n",
              "max           5.935761\n",
              "Name: viral_selection, dtype: float64"
            ]
          },
          "execution_count": 21,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "gas1['viral_selection'] = numpy.log2(gas1.virus_F / gas1.plasmid_F)\n",
        "assert 0 == gas1.viral_selection.isna().sum()\n",
        "assert not numpy.any(numpy.isinf(gas1.viral_selection))\n",
        "gas1.viral_selection.describe()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qlVxyU9eHwsD"
      },
      "source": [
        "### Compute GMM threshold"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "executionInfo": {
          "elapsed": 4782,
          "status": "ok",
          "timestamp": 1593126599459,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "we0lrh3OIrXx",
        "outputId": "98de5bba-8766-4b1e-d81b-fac833b4cece"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "selection coeff cutoff = -2.811\n",
            "0.5094091595202946\n"
          ]
        }
      ],
      "source": [
        "# Classify the selection coeff series after fitting to a GMM\n",
        "gmm_model = TwoGaussianMixtureModelLabeler(\n",
        "    gas1[gas1.partition.isin(['stop', 'wild_type'])].viral_selection)\n",
        "gas1['viral_selection_gmm'] = gmm_model.predict(gas1.viral_selection)\n",
        "\n",
        "# Compute the threshold for the viable class from the GMM labels\n",
        "selection_coeff_threshold = gas1.loc[gas1.viral_selection_gmm == 2, 'viral_selection'].min()\n",
        "print('selection coeff cutoff = %.3f' % selection_coeff_threshold)\n",
        "\n",
        "# Add a label column\n",
        "def is_viable_mutant(mutant_data):\n",
        "  return mutant_data['viral_selection'] \u003e selection_coeff_threshold\n",
        "gas1['is_viable'] = gas1.apply(is_viable_mutant, axis=1)\n",
        "\n",
        "print(gas1.is_viable.mean())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "crcRQniSbCod"
      },
      "source": [
        "----"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jZ_aMJuDyS-N"
      },
      "source": [
        "### De-dupe model-designed sequences"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YK8Hr6QTcEUT"
      },
      "source": [
        "#### Partition the sequences that should not be de-deduped\n",
        "\n",
        "Split off the partitions for which we want to retain replicates, such as controls/etc.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 336
        },
        "executionInfo": {
          "elapsed": 414,
          "status": "ok",
          "timestamp": 1593126604927,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "MmYz3pKSyXTS",
        "outputId": "1e6f384f-3c10-4f42-d823-5a69acd1e428"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "rnn_standard_walked                     21337\n",
              "cnn_designed_plus_rand_train_walked     21251\n",
              "rnn_designed_plus_rand_train_walked     21218\n",
              "lr_standard_walked                      20951\n",
              "cnn_rand_doubles_plus_single_walked     20936\n",
              "cnn_standard_walked                     20878\n",
              "rnn_rand_doubles_plus_singles_walked    20623\n",
              "lr_rand_doubles_plus_single_walked      20473\n",
              "lr_designed_plus_rand_train_walked      20164\n",
              "lr_rand_doubles_plus_single_seed         2071\n",
              "rnn_designed_plus_rand_train_seed        2065\n",
              "rnn_rand_doubles_plus_singles_seed       2045\n",
              "lr_designed_plus_rand_train_seed         2030\n",
              "cnn_rand_doubles_plus_single_seed        2022\n",
              "lr_standard_seed                         1989\n",
              "cnn_standard_seed                        1924\n",
              "rnn_standard_seed                        1916\n",
              "cnn_designed_plus_rand_train_seed        1898\n",
              "Name: partition, dtype: int64"
            ]
          },
          "execution_count": 24,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "ml_generated_seqs = [\n",
        "    'cnn_designed_plus_rand_train_seed',\n",
        "    'cnn_designed_plus_rand_train_walked',\n",
        "    'cnn_rand_doubles_plus_single_seed',\n",
        "    'cnn_rand_doubles_plus_single_walked',\n",
        "    'cnn_standard_seed',\n",
        "    'cnn_standard_walked',\n",
        "    'lr_designed_plus_rand_train_seed',\n",
        "    'lr_designed_plus_rand_train_walked',\n",
        "    'lr_rand_doubles_plus_single_seed',\n",
        "    'lr_rand_doubles_plus_single_walked',\n",
        "    'lr_standard_seed',\n",
        "    'lr_standard_walked',\n",
        "    'rnn_designed_plus_rand_train_seed',\n",
        "    'rnn_designed_plus_rand_train_walked',\n",
        "    'rnn_rand_doubles_plus_singles_seed',\n",
        "    'rnn_rand_doubles_plus_singles_walked',\n",
        "    'rnn_standard_seed',\n",
        "    'rnn_standard_walked',\n",
        "]\n",
        "\n",
        "is_ml_generated_mask = gas1.partition.isin(ml_generated_seqs)\n",
        "ml_gen_df = gas1[is_ml_generated_mask].copy()\n",
        "non_ml_gen_df = gas1[~is_ml_generated_mask].copy()\n",
        "\n",
        "ml_gen_df.partition.value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xELIUhWR6ysl"
      },
      "outputs": [],
      "source": [
        "ml_gen_deduped = ml_gen_df.groupby('sequence').apply(\n",
        "    lambda dupes: dupes.loc[dupes.plasmid_N.idxmax()]).copy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 362
        },
        "executionInfo": {
          "elapsed": 362,
          "status": "ok",
          "timestamp": 1593126768681,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "n6yS2gEZGa8U",
        "outputId": "3b972a1d-b54e-4857-ae52-f65e1d4d904f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(201426, 65)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "\u003cdiv\u003e\n",
              "\u003cstyle scoped\u003e\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "\u003c/style\u003e\n",
              "\u003ctable border=\"1\" class=\"dataframe\"\u003e\n",
              "  \u003cthead\u003e\n",
              "    \u003ctr style=\"text-align: right;\"\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003eUnnamed: 0\u003c/th\u003e\n",
              "      \u003cth\u003eUnnamed: 0.1\u003c/th\u003e\n",
              "      \u003cth\u003esequence\u003c/th\u003e\n",
              "      \u003cth\u003epartition\u003c/th\u003e\n",
              "      \u003cth\u003echip\u003c/th\u003e\n",
              "      \u003cth\u003econtrol\u003c/th\u003e\n",
              "      \u003cth\u003eis_wt_aa\u003c/th\u003e\n",
              "      \u003cth\u003eis_wt_nt\u003c/th\u003e\n",
              "      \u003cth\u003emutation_sequence\u003c/th\u003e\n",
              "      \u003cth\u003enum_mutations\u003c/th\u003e\n",
              "      \u003cth\u003erep_i\u003c/th\u003e\n",
              "      \u003cth\u003erep_original\u003c/th\u003e\n",
              "      \u003cth\u003erep_total\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1a_plasmid_x\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1b_plasmid_x\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1c_plasmid_x\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1d_plasmid_x\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v3_rep1a_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v3_rep1b_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v3_rep1c_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v3_rep1d_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v4_rep2a_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v4_rep2b_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v4_rep2c_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v4_rep2d_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v5_rep3a_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_v5_rep3b_virus\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1a_plasmid_y\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1b_plasmid_y\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1c_plasmid_y\u003c/th\u003e\n",
              "      \u003cth\u003eEK269_GAS1_p1_rep1d_plasmid_y\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v3_rep1a_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v3_rep1b_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v3_rep1c_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v3_rep1d_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v4_rep2a_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v4_rep2b_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v4_rep2c_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v4_rep2d_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v5_rep3a_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v5_rep3b_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v5_rep3c_virus\u003c/th\u003e\n",
              "      \u003cth\u003eold_EK269_GAS1_v5_rep3d_virus\u003c/th\u003e\n",
              "      \u003cth\u003eGAS1_plasmid_N\u003c/th\u003e\n",
              "      \u003cth\u003ev0_GAS1_plasmid_N\u003c/th\u003e\n",
              "      \u003cth\u003ev1_GAS1_plasmid_N\u003c/th\u003e\n",
              "      \u003cth\u003ev0_GAS1_virus_N\u003c/th\u003e\n",
              "      \u003cth\u003ev1_GAS1_virus_N\u003c/th\u003e\n",
              "      \u003cth\u003eGAS1_virus_N\u003c/th\u003e\n",
              "      \u003cth\u003ev0_GAS1_plasmid_F\u003c/th\u003e\n",
              "      \u003cth\u003ev1_GAS1_plasmid_F\u003c/th\u003e\n",
              "      \u003cth\u003eGAS1_plasmid_F\u003c/th\u003e\n",
              "      \u003cth\u003ev0_GAS1_virus_F\u003c/th\u003e\n",
              "      \u003cth\u003ev1_GAS1_virus_F\u003c/th\u003e\n",
              "      \u003cth\u003eGAS1_virus_F\u003c/th\u003e\n",
              "      \u003cth\u003ev0_GAS1_virus_S\u003c/th\u003e\n",
              "      \u003cth\u003ev1_GAS1_virus_S\u003c/th\u003e\n",
              "      \u003cth\u003eGAS1_virus_S\u003c/th\u003e\n",
              "      \u003cth\u003evirus_N\u003c/th\u003e\n",
              "      \u003cth\u003eplasmid_N\u003c/th\u003e\n",
              "      \u003cth\u003evirus_F\u003c/th\u003e\n",
              "      \u003cth\u003eplasmid_F\u003c/th\u003e\n",
              "      \u003cth\u003eviral_selection\u003c/th\u003e\n",
              "      \u003cth\u003eviral_selection_gmm\u003c/th\u003e\n",
              "      \u003cth\u003eis_viable\u003c/th\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003esequence\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "    \u003c/tr\u003e\n",
              "  \u003c/thead\u003e\n",
              "  \u003ctbody\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003eAAEEIATTNPVATEQYGSVcAaNmGEeApDaQaEgGd\u003c/th\u003e\n",
              "      \u003ctd\u003e82181\u003c/td\u003e\n",
              "      \u003ctd\u003eGCAGCCGAGGAAATCGCTACAACCAATCCCGTGGCTACGGAGCAGT...\u003c/td\u003e\n",
              "      \u003ctd\u003eAAEEIATTNPVATEQYGSVcAaNmGEeApDaQaEgGd\u003c/td\u003e\n",
              "      \u003ctd\u003ecnn_designed_plus_rand_train_walked\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003eAA___A_____________cAaNmGEeApDaQaEgGd\u003c/td\u003e\n",
              "      \u003ctd\u003e21\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e36\u003c/td\u003e\n",
              "      \u003ctd\u003e129\u003c/td\u003e\n",
              "      \u003ctd\u003e94\u003c/td\u003e\n",
              "      \u003ctd\u003e91\u003c/td\u003e\n",
              "      \u003ctd\u003e322\u003c/td\u003e\n",
              "      \u003ctd\u003e94\u003c/td\u003e\n",
              "      \u003ctd\u003e94\u003c/td\u003e\n",
              "      \u003ctd\u003e378\u003c/td\u003e\n",
              "      \u003ctd\u003e277\u003c/td\u003e\n",
              "      \u003ctd\u003e557\u003c/td\u003e\n",
              "      \u003ctd\u003e277\u003c/td\u003e\n",
              "      \u003ctd\u003e87\u003c/td\u003e\n",
              "      \u003ctd\u003e61\u003c/td\u003e\n",
              "      \u003ctd\u003e63\u003c/td\u003e\n",
              "      \u003ctd\u003e14.0\u003c/td\u003e\n",
              "      \u003ctd\u003e15.0\u003c/td\u003e\n",
              "      \u003ctd\u003e15.0\u003c/td\u003e\n",
              "      \u003ctd\u003e12.0\u003c/td\u003e\n",
              "      \u003ctd\u003e19.0\u003c/td\u003e\n",
              "      \u003ctd\u003e35.0\u003c/td\u003e\n",
              "      \u003ctd\u003e23.0\u003c/td\u003e\n",
              "      \u003ctd\u003e26.0\u003c/td\u003e\n",
              "      \u003ctd\u003e43.0\u003c/td\u003e\n",
              "      \u003ctd\u003e35.0\u003c/td\u003e\n",
              "      \u003ctd\u003e31.0\u003c/td\u003e\n",
              "      \u003ctd\u003e11.0\u003c/td\u003e\n",
              "      \u003ctd\u003e21.0\u003c/td\u003e\n",
              "      \u003ctd\u003e29.0\u003c/td\u003e\n",
              "      \u003ctd\u003e32.0\u003c/td\u003e\n",
              "      \u003ctd\u003e18.0\u003c/td\u003e\n",
              "      \u003ctd\u003e406.0\u003c/td\u003e\n",
              "      \u003ctd\u003e56.0\u003c/td\u003e\n",
              "      \u003ctd\u003e350\u003c/td\u003e\n",
              "      \u003ctd\u003e323.0\u003c/td\u003e\n",
              "      \u003ctd\u003e2210\u003c/td\u003e\n",
              "      \u003ctd\u003e2533.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000008\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000006\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000006\u003c/td\u003e\n",
              "      \u003ctd\u003e1.025716e-05\u003c/td\u003e\n",
              "      \u003ctd\u003e1.163842e-05\u003c/td\u003e\n",
              "      \u003ctd\u003e1.144194e-05\u003c/td\u003e\n",
              "      \u003ctd\u003e0.444881\u003c/td\u003e\n",
              "      \u003ctd\u003e0.967444\u003c/td\u003e\n",
              "      \u003ctd\u003e0.900450\u003c/td\u003e\n",
              "      \u003ctd\u003e2534.0\u003c/td\u003e\n",
              "      \u003ctd\u003e407.0\u003c/td\u003e\n",
              "      \u003ctd\u003e1.143470e-05\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000006\u003c/td\u003e\n",
              "      \u003ctd\u003e0.900998\u003c/td\u003e\n",
              "      \u003ctd\u003e2\u003c/td\u003e\n",
              "      \u003ctd\u003eTrue\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003eAAEEIATTNPVAtYAQWGCnTAgNnGLTtAeTgNlDs\u003c/th\u003e\n",
              "      \u003ctd\u003e85396\u003c/td\u003e\n",
              "      \u003ctd\u003eGCGGCTGAGGAAATCGCGACAACCAATCCCGTGGCTACCTATGCTC...\u003c/td\u003e\n",
              "      \u003ctd\u003eAAEEIATTNPVAtYAQWGCnTAgNnGLTtAeTgNlDs\u003c/td\u003e\n",
              "      \u003ctd\u003ecnn_designed_plus_rand_train_walked\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003eAA___A______tYA_W_CnTAgNnG_TtAeTg_lDs\u003c/td\u003e\n",
              "      \u003ctd\u003e24\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e20\u003c/td\u003e\n",
              "      \u003ctd\u003e118\u003c/td\u003e\n",
              "      \u003ctd\u003e79\u003c/td\u003e\n",
              "      \u003ctd\u003e68\u003c/td\u003e\n",
              "      \u003ctd\u003e4\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e6\u003c/td\u003e\n",
              "      \u003ctd\u003e8\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e3\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e6.0\u003c/td\u003e\n",
              "      \u003ctd\u003e4.0\u003c/td\u003e\n",
              "      \u003ctd\u003e8.0\u003c/td\u003e\n",
              "      \u003ctd\u003e6.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e1.0\u003c/td\u003e\n",
              "      \u003ctd\u003e1.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e309.0\u003c/td\u003e\n",
              "      \u003ctd\u003e24.0\u003c/td\u003e\n",
              "      \u003ctd\u003e285\u003c/td\u003e\n",
              "      \u003ctd\u003e2.0\u003c/td\u003e\n",
              "      \u003ctd\u003e33\u003c/td\u003e\n",
              "      \u003ctd\u003e35.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000003\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000005\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000005\u003c/td\u003e\n",
              "      \u003ctd\u003e6.351180e-08\u003c/td\u003e\n",
              "      \u003ctd\u003e1.737864e-07\u003c/td\u003e\n",
              "      \u003ctd\u003e1.581002e-07\u003c/td\u003e\n",
              "      \u003ctd\u003e-5.668117\u003c/td\u003e\n",
              "      \u003ctd\u003e-4.801599\u003c/td\u003e\n",
              "      \u003ctd\u003e-4.883025\u003c/td\u003e\n",
              "      \u003ctd\u003e36.0\u003c/td\u003e\n",
              "      \u003ctd\u003e310.0\u003c/td\u003e\n",
              "      \u003ctd\u003e1.624504e-07\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000005\u003c/td\u003e\n",
              "      \u003ctd\u003e-4.843517\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003eFalse\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003eAAEEIFTTNPEALEQVwASCSeRnPfQhLGsPEI\u003c/th\u003e\n",
              "      \u003ctd\u003e172406\u003c/td\u003e\n",
              "      \u003ctd\u003eGCTGCGGAGGAAATCTTTACAACCAATCCCGAGGCTCTCGAGCAGG...\u003c/td\u003e\n",
              "      \u003ctd\u003eAAEEIFTTNPEALEQVwASCSeRnPfQhLGsPEI\u003c/td\u003e\n",
              "      \u003ctd\u003elr_rand_doubles_plus_single_walked\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003eAA___F____E_L__VwA_C_eRnPfQhLGsPEI\u003c/td\u003e\n",
              "      \u003ctd\u003e22\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e17\u003c/td\u003e\n",
              "      \u003ctd\u003e67\u003c/td\u003e\n",
              "      \u003ctd\u003e44\u003c/td\u003e\n",
              "      \u003ctd\u003e51\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e3\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e6.0\u003c/td\u003e\n",
              "      \u003ctd\u003e2.0\u003c/td\u003e\n",
              "      \u003ctd\u003e6.0\u003c/td\u003e\n",
              "      \u003ctd\u003e5.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e1.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e198.0\u003c/td\u003e\n",
              "      \u003ctd\u003e19.0\u003c/td\u003e\n",
              "      \u003ctd\u003e179\u003c/td\u003e\n",
              "      \u003ctd\u003e1.0\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e6.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000003\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000003\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000003\u003c/td\u003e\n",
              "      \u003ctd\u003e3.175590e-08\u003c/td\u003e\n",
              "      \u003ctd\u003e2.633127e-08\u003c/td\u003e\n",
              "      \u003ctd\u003e2.710290e-08\u003c/td\u003e\n",
              "      \u003ctd\u003e-6.331082\u003c/td\u003e\n",
              "      \u003ctd\u003e-6.853063\u003c/td\u003e\n",
              "      \u003ctd\u003e-6.785239\u003c/td\u003e\n",
              "      \u003ctd\u003e7.0\u003c/td\u003e\n",
              "      \u003ctd\u003e199.0\u003c/td\u003e\n",
              "      \u003ctd\u003e3.158758e-08\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000003\u003c/td\u003e\n",
              "      \u003ctd\u003e-6.566587\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003eFalse\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003eAAEEIHPTNPEALEQViASCSeRnPfQhLGGEaH\u003c/th\u003e\n",
              "      \u003ctd\u003e172429\u003c/td\u003e\n",
              "      \u003ctd\u003eGCCGCGGAGGAAATCCACCCGACCAATCCCGAAGCTCTTGAGCAGG...\u003c/td\u003e\n",
              "      \u003ctd\u003eAAEEIHPTNPEALEQViASCSeRnPfQhLGGEaH\u003c/td\u003e\n",
              "      \u003ctd\u003elr_rand_doubles_plus_single_walked\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003eAA___HP___E_L__ViA_C_eRnPfQhLG_EaH\u003c/td\u003e\n",
              "      \u003ctd\u003e22\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e35\u003c/td\u003e\n",
              "      \u003ctd\u003e245\u003c/td\u003e\n",
              "      \u003ctd\u003e181\u003c/td\u003e\n",
              "      \u003ctd\u003e150\u003c/td\u003e\n",
              "      \u003ctd\u003e2\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e2\u003c/td\u003e\n",
              "      \u003ctd\u003e2\u003c/td\u003e\n",
              "      \u003ctd\u003e2\u003c/td\u003e\n",
              "      \u003ctd\u003e3\u003c/td\u003e\n",
              "      \u003ctd\u003e3\u003c/td\u003e\n",
              "      \u003ctd\u003e2\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e19.0\u003c/td\u003e\n",
              "      \u003ctd\u003e23.0\u003c/td\u003e\n",
              "      \u003ctd\u003e27.0\u003c/td\u003e\n",
              "      \u003ctd\u003e11.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e1.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e1.0\u003c/td\u003e\n",
              "      \u003ctd\u003e2.0\u003c/td\u003e\n",
              "      \u003ctd\u003e691.0\u003c/td\u003e\n",
              "      \u003ctd\u003e80.0\u003c/td\u003e\n",
              "      \u003ctd\u003e611\u003c/td\u003e\n",
              "      \u003ctd\u003e4.0\u003c/td\u003e\n",
              "      \u003ctd\u003e18\u003c/td\u003e\n",
              "      \u003ctd\u003e22.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000011\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000010\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000010\u003c/td\u003e\n",
              "      \u003ctd\u003e1.270236e-07\u003c/td\u003e\n",
              "      \u003ctd\u003e9.479256e-08\u003c/td\u003e\n",
              "      \u003ctd\u003e9.937730e-08\u003c/td\u003e\n",
              "      \u003ctd\u003e-6.405082\u003c/td\u003e\n",
              "      \u003ctd\u003e-6.776279\u003c/td\u003e\n",
              "      \u003ctd\u003e-6.713956\u003c/td\u003e\n",
              "      \u003ctd\u003e23.0\u003c/td\u003e\n",
              "      \u003ctd\u003e692.0\u003c/td\u003e\n",
              "      \u003ctd\u003e1.037878e-07\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000010\u003c/td\u003e\n",
              "      \u003ctd\u003e-6.648384\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003eFalse\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003eAAEEIHPTNPEALEQVwASCAeRnPfQhLGGEI\u003c/th\u003e\n",
              "      \u003ctd\u003e172420\u003c/td\u003e\n",
              "      \u003ctd\u003eGCGGCTGAGGAAATCCATCCAACCAATCCCGAAGCGCTAGAGCAGG...\u003c/td\u003e\n",
              "      \u003ctd\u003eAAEEIHPTNPEALEQVwASCAeRnPfQhLGGEI\u003c/td\u003e\n",
              "      \u003ctd\u003elr_rand_doubles_plus_single_walked\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003eAA___HP___E_L__VwA_CAeRnPfQhLG_EI\u003c/td\u003e\n",
              "      \u003ctd\u003e22\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e42\u003c/td\u003e\n",
              "      \u003ctd\u003e204\u003c/td\u003e\n",
              "      \u003ctd\u003e143\u003c/td\u003e\n",
              "      \u003ctd\u003e130\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e3\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e3\u003c/td\u003e\n",
              "      \u003ctd\u003e2\u003c/td\u003e\n",
              "      \u003ctd\u003e2\u003c/td\u003e\n",
              "      \u003ctd\u003e1\u003c/td\u003e\n",
              "      \u003ctd\u003e17.0\u003c/td\u003e\n",
              "      \u003ctd\u003e19.0\u003c/td\u003e\n",
              "      \u003ctd\u003e16.0\u003c/td\u003e\n",
              "      \u003ctd\u003e10.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e1.0\u003c/td\u003e\n",
              "      \u003ctd\u003e1.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.0\u003c/td\u003e\n",
              "      \u003ctd\u003e581.0\u003c/td\u003e\n",
              "      \u003ctd\u003e62.0\u003c/td\u003e\n",
              "      \u003ctd\u003e519\u003c/td\u003e\n",
              "      \u003ctd\u003e2.0\u003c/td\u003e\n",
              "      \u003ctd\u003e28\u003c/td\u003e\n",
              "      \u003ctd\u003e30.0\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000008\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000009\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000009\u003c/td\u003e\n",
              "      \u003ctd\u003e6.351180e-08\u003c/td\u003e\n",
              "      \u003ctd\u003e1.474551e-07\u003c/td\u003e\n",
              "      \u003ctd\u003e1.355145e-07\u003c/td\u003e\n",
              "      \u003ctd\u003e-7.037351\u003c/td\u003e\n",
              "      \u003ctd\u003e-5.903411\u003c/td\u003e\n",
              "      \u003ctd\u003e-6.016349\u003c/td\u003e\n",
              "      \u003ctd\u003e31.0\u003c/td\u003e\n",
              "      \u003ctd\u003e582.0\u003c/td\u003e\n",
              "      \u003ctd\u003e1.398879e-07\u003c/td\u003e\n",
              "      \u003ctd\u003e0.000009\u003c/td\u003e\n",
              "      \u003ctd\u003e-5.967996\u003c/td\u003e\n",
              "      \u003ctd\u003e0\u003c/td\u003e\n",
              "      \u003ctd\u003eFalse\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "  \u003c/tbody\u003e\n",
              "\u003c/table\u003e\n",
              "\u003c/div\u003e"
            ],
            "text/plain": [
              "                                       Unnamed: 0  ... is_viable\n",
              "sequence                                           ...          \n",
              "AAEEIATTNPVATEQYGSVcAaNmGEeApDaQaEgGd       82181  ...      True\n",
              "AAEEIATTNPVAtYAQWGCnTAgNnGLTtAeTgNlDs       85396  ...     False\n",
              "AAEEIFTTNPEALEQVwASCSeRnPfQhLGsPEI         172406  ...     False\n",
              "AAEEIHPTNPEALEQViASCSeRnPfQhLGGEaH         172429  ...     False\n",
              "AAEEIHPTNPEALEQVwASCAeRnPfQhLGGEI          172420  ...     False\n",
              "\n",
              "[5 rows x 65 columns]"
            ]
          },
          "execution_count": 26,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "display(ml_gen_deduped.shape)\n",
        "ml_gen_deduped.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oZGKy5r0dg7D"
      },
      "source": [
        "#### Concatenate de-deduped ML-generated seqs with rest"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 454
        },
        "executionInfo": {
          "elapsed": 628,
          "status": "ok",
          "timestamp": 1593126768958,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "s-dC1aT-dgd_",
        "outputId": "ff66f635-4a39-41d7-d911-32197e49e965"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(236783, 65)\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "random_doubles                          25040\n",
              "rnn_standard_walked                     20838\n",
              "cnn_designed_plus_rand_train_walked     20759\n",
              "rnn_designed_plus_rand_train_walked     20731\n",
              "lr_standard_walked                      20456\n",
              "cnn_rand_doubles_plus_single_walked     20454\n",
              "cnn_standard_walked                     20395\n",
              "rnn_rand_doubles_plus_singles_walked    20154\n",
              "lr_rand_doubles_plus_single_walked      19999\n",
              "lr_designed_plus_rand_train_walked      19680\n",
              "singles                                  3952\n",
              "previous_chip_viable                     2997\n",
              "previous_chip_nonviable                  2997\n",
              "lr_rand_doubles_plus_single_seed         2071\n",
              "rnn_designed_plus_rand_train_seed        2065\n",
              "rnn_rand_doubles_plus_singles_seed       2045\n",
              "lr_designed_plus_rand_train_seed         2030\n",
              "cnn_rand_doubles_plus_single_seed        2022\n",
              "lr_standard_seed                         1989\n",
              "cnn_standard_seed                        1924\n",
              "rnn_standard_seed                        1916\n",
              "cnn_designed_plus_rand_train_seed        1898\n",
              "wild_type                                 200\n",
              "stop                                      171\n",
              "Name: partition, dtype: int64"
            ]
          },
          "execution_count": 27,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "gas1_deduped = pandas.concat([ml_gen_deduped, non_ml_gen_df], axis=0)\n",
        "print(gas1_deduped.shape)\n",
        "gas1_deduped.partition.value_counts()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3ACPVRHCV1sq"
      },
      "source": [
        "## Compute edit distance for chip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xg2CYqXDIKqy"
      },
      "outputs": [],
      "source": [
        "gas1 = gas1_deduped"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        },
        "executionInfo": {
          "elapsed": 917,
          "status": "ok",
          "timestamp": 1593126819825,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "0VmmyNdVV4Fp",
        "outputId": "a647f198-67ab-4b5a-8d31-49bd4b122620"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "count    236783.000000\n",
              "mean         13.563174\n",
              "std           7.489709\n",
              "min           0.000000\n",
              "25%           7.000000\n",
              "50%          14.000000\n",
              "75%          20.000000\n",
              "max          33.000000\n",
              "Name: num_edits, dtype: float64"
            ]
          },
          "execution_count": 31,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "gas1['num_edits'] = gas1.sequence.apply(\n",
        "    lambda s: Levenshtein.distance(R1_TILE21_WT_SEQ, s))\n",
        "gas1.num_edits.describe()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_tvwnyvQHppW"
      },
      "outputs": [],
      "source": [
        "COLUMN_SCHEMA = [ \n",
        "    'sequence',\n",
        "    'partition',\n",
        "    'mutation_sequence',\n",
        "    'num_mutations',\n",
        "    'num_edits',    \n",
        "    'viral_selection',\n",
        "    'is_viable',\n",
        "]\n",
        "\n",
        "gas1a = gas1[COLUMN_SCHEMA].copy()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W7bF-d67Y5E1"
      },
      "source": [
        "#### Concat with training data chip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "executionInfo": {
          "elapsed": 929,
          "status": "ok",
          "timestamp": 1593126937032,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "qE6NK7fKtwp4",
        "outputId": "5922be09-4734-450c-b5b1-d3d5c8ca6b05"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\u003cdiv\u003e\n",
              "\u003cstyle scoped\u003e\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "\u003c/style\u003e\n",
              "\u003ctable border=\"1\" class=\"dataframe\"\u003e\n",
              "  \u003cthead\u003e\n",
              "    \u003ctr style=\"text-align: right;\"\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003esequence\u003c/th\u003e\n",
              "      \u003cth\u003epartition\u003c/th\u003e\n",
              "      \u003cth\u003emutation_sequence\u003c/th\u003e\n",
              "      \u003cth\u003enum_mutations\u003c/th\u003e\n",
              "      \u003cth\u003enum_edits\u003c/th\u003e\n",
              "      \u003cth\u003eviral_selection\u003c/th\u003e\n",
              "      \u003cth\u003eis_viable\u003c/th\u003e\n",
              "    \u003c/tr\u003e\n",
              "  \u003c/thead\u003e\n",
              "  \u003ctbody\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e0\u003c/th\u003e\n",
              "      \u003ctd\u003eADEEIRATNPIATEMYGSVSTNLQLGNR\u003c/td\u003e\n",
              "      \u003ctd\u003edesigned\u003c/td\u003e\n",
              "      \u003ctd\u003eAD____A___I___M_________L___\u003c/td\u003e\n",
              "      \u003ctd\u003e6\u003c/td\u003e\n",
              "      \u003ctd\u003e6\u003c/td\u003e\n",
              "      \u003ctd\u003e-2.027259\u003c/td\u003e\n",
              "      \u003ctd\u003eFalse\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e1\u003c/th\u003e\n",
              "      \u003ctd\u003eADEEIRATNPVATEQYGSVSTNQQRQNR\u003c/td\u003e\n",
              "      \u003ctd\u003edesigned\u003c/td\u003e\n",
              "      \u003ctd\u003eAD____A_______________Q__Q__\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e-0.429554\u003c/td\u003e\n",
              "      \u003ctd\u003eTrue\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e2\u003c/th\u003e\n",
              "      \u003ctd\u003eADEEIRTTNPVATEQWGGVSTNLQIGNY\u003c/td\u003e\n",
              "      \u003ctd\u003edesigned\u003c/td\u003e\n",
              "      \u003ctd\u003eAD_____________W_G______I__Y\u003c/td\u003e\n",
              "      \u003ctd\u003e6\u003c/td\u003e\n",
              "      \u003ctd\u003e6\u003c/td\u003e\n",
              "      \u003ctd\u003e-0.527843\u003c/td\u003e\n",
              "      \u003ctd\u003eTrue\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "  \u003c/tbody\u003e\n",
              "\u003c/table\u003e\n",
              "\u003c/div\u003e"
            ],
            "text/plain": [
              "                       sequence partition  ... viral_selection  is_viable\n",
              "0  ADEEIRATNPIATEMYGSVSTNLQLGNR  designed  ...       -2.027259      False\n",
              "1  ADEEIRATNPVATEQYGSVSTNQQRQNR  designed  ...       -0.429554       True\n",
              "2  ADEEIRTTNPVATEQWGGVSTNLQIGNY  designed  ...       -0.527843       True\n",
              "\n",
              "[3 rows x 7 columns]"
            ]
          },
          "execution_count": 37,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harvard = pandas.read_csv('r0r1_with_partitions_and_labels.csv', index_col=None)\n",
        "\n",
        "harvard = harvard.rename({\n",
        "    'S': 'viral_selection',\n",
        "    'aa_seq': 'sequence',\n",
        "    'mask': 'mutation_sequence',\n",
        "    'mut': 'num_mutations',\n",
        "}, axis=1)\n",
        "\n",
        "designed_mask = harvard.partition.isin(['min_fit', 'thresh', 'temp'])\n",
        "harvard.loc[designed_mask, ['partition']] = 'designed'\n",
        "\n",
        "harvard['num_edits'] = harvard.sequence.apply(\n",
        "    lambda s: Levenshtein.distance(R1_TILE21_WT_SEQ, s))\n",
        "harvard.num_edits.describe()\n",
        "\n",
        "harvard1 = harvard[COLUMN_SCHEMA].copy()\n",
        "\n",
        "harvard1.head(3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 504
        },
        "executionInfo": {
          "elapsed": 445,
          "status": "ok",
          "timestamp": 1593126945342,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "09Cw43etY4Vo",
        "outputId": "89c210fc-30ca-47c9-f854-da221929eb28"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(304152, 8)\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "designed                                56372\n",
              "random_doubles                          25040\n",
              "rnn_standard_walked                     20838\n",
              "cnn_designed_plus_rand_train_walked     20759\n",
              "rnn_designed_plus_rand_train_walked     20731\n",
              "lr_standard_walked                      20456\n",
              "cnn_rand_doubles_plus_single_walked     20454\n",
              "cnn_standard_walked                     20395\n",
              "rnn_rand_doubles_plus_singles_walked    20154\n",
              "lr_rand_doubles_plus_single_walked      19999\n",
              "lr_designed_plus_rand_train_walked      19680\n",
              "rand                                     9885\n",
              "singles                                  3952\n",
              "previous_chip_viable                     2997\n",
              "previous_chip_nonviable                  2997\n",
              "lr_rand_doubles_plus_single_seed         2071\n",
              "rnn_designed_plus_rand_train_seed        2065\n",
              "rnn_rand_doubles_plus_singles_seed       2045\n",
              "lr_designed_plus_rand_train_seed         2030\n",
              "cnn_rand_doubles_plus_single_seed        2022\n",
              "lr_standard_seed                         1989\n",
              "cnn_standard_seed                        1924\n",
              "rnn_standard_seed                        1916\n",
              "cnn_designed_plus_rand_train_seed        1898\n",
              "single                                   1112\n",
              "wild_type                                 200\n",
              "stop                                      171\n",
              "Name: partition, dtype: int64"
            ]
          },
          "execution_count": 38,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harvard1['chip'] = 'harvard'\n",
        "gas1a['chip'] = 'gas1'\n",
        "\n",
        "combined = pandas.concat([\n",
        "    harvard1, \n",
        "    gas1a,\n",
        "], axis=0, sort=False)\n",
        "print(combined.shape)\n",
        "combined.partition.value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "executionInfo": {
          "elapsed": 378,
          "status": "ok",
          "timestamp": 1593126948316,
          "user": {
            "displayName": "Lucy Colwell",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhDVS9VL55nJ_BMyjGqE4RYboOf4i5N9LaE148oGg=s64",
            "userId": "17590064334699832966"
          },
          "user_tz": 240
        },
        "id": "sDi7oEM2nsL7",
        "outputId": "3be1645b-44a8-40c6-b60a-861bedf2a995"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\u003cdiv\u003e\n",
              "\u003cstyle scoped\u003e\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "\u003c/style\u003e\n",
              "\u003ctable border=\"1\" class=\"dataframe\"\u003e\n",
              "  \u003cthead\u003e\n",
              "    \u003ctr style=\"text-align: right;\"\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003esequence\u003c/th\u003e\n",
              "      \u003cth\u003epartition\u003c/th\u003e\n",
              "      \u003cth\u003emutation_sequence\u003c/th\u003e\n",
              "      \u003cth\u003enum_mutations\u003c/th\u003e\n",
              "      \u003cth\u003enum_edits\u003c/th\u003e\n",
              "      \u003cth\u003eviral_selection\u003c/th\u003e\n",
              "      \u003cth\u003eis_viable\u003c/th\u003e\n",
              "      \u003cth\u003echip\u003c/th\u003e\n",
              "    \u003c/tr\u003e\n",
              "  \u003c/thead\u003e\n",
              "  \u003ctbody\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e0\u003c/th\u003e\n",
              "      \u003ctd\u003eADEEIRATNPIATEMYGSVSTNLQLGNR\u003c/td\u003e\n",
              "      \u003ctd\u003edesigned\u003c/td\u003e\n",
              "      \u003ctd\u003eAD____A___I___M_________L___\u003c/td\u003e\n",
              "      \u003ctd\u003e6\u003c/td\u003e\n",
              "      \u003ctd\u003e6\u003c/td\u003e\n",
              "      \u003ctd\u003e-2.027259\u003c/td\u003e\n",
              "      \u003ctd\u003eFalse\u003c/td\u003e\n",
              "      \u003ctd\u003eharvard\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e1\u003c/th\u003e\n",
              "      \u003ctd\u003eADEEIRATNPVATEQYGSVSTNQQRQNR\u003c/td\u003e\n",
              "      \u003ctd\u003edesigned\u003c/td\u003e\n",
              "      \u003ctd\u003eAD____A_______________Q__Q__\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e5\u003c/td\u003e\n",
              "      \u003ctd\u003e-0.429554\u003c/td\u003e\n",
              "      \u003ctd\u003eTrue\u003c/td\u003e\n",
              "      \u003ctd\u003eharvard\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e2\u003c/th\u003e\n",
              "      \u003ctd\u003eADEEIRTTNPVATEQWGGVSTNLQIGNY\u003c/td\u003e\n",
              "      \u003ctd\u003edesigned\u003c/td\u003e\n",
              "      \u003ctd\u003eAD_____________W_G______I__Y\u003c/td\u003e\n",
              "      \u003ctd\u003e6\u003c/td\u003e\n",
              "      \u003ctd\u003e6\u003c/td\u003e\n",
              "      \u003ctd\u003e-0.527843\u003c/td\u003e\n",
              "      \u003ctd\u003eTrue\u003c/td\u003e\n",
              "      \u003ctd\u003eharvard\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e3\u003c/th\u003e\n",
              "      \u003ctd\u003eADEEIRTTNPVATEQYGEVSTNLQRGNR\u003c/td\u003e\n",
              "      \u003ctd\u003edesigned\u003c/td\u003e\n",
              "      \u003ctd\u003eAD_______________E__________\u003c/td\u003e\n",
              "      \u003ctd\u003e3\u003c/td\u003e\n",
              "      \u003ctd\u003e3\u003c/td\u003e\n",
              "      \u003ctd\u003e2.887908\u003c/td\u003e\n",
              "      \u003ctd\u003eTrue\u003c/td\u003e\n",
              "      \u003ctd\u003eharvard\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e4\u003c/th\u003e\n",
              "      \u003ctd\u003eADEEIRTTNPVATEQYGSVSTNLQRGNR\u003c/td\u003e\n",
              "      \u003ctd\u003edesigned\u003c/td\u003e\n",
              "      \u003ctd\u003eAD__________________________\u003c/td\u003e\n",
              "      \u003ctd\u003e2\u003c/td\u003e\n",
              "      \u003ctd\u003e2\u003c/td\u003e\n",
              "      \u003ctd\u003e0.575730\u003c/td\u003e\n",
              "      \u003ctd\u003eTrue\u003c/td\u003e\n",
              "      \u003ctd\u003eharvard\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "  \u003c/tbody\u003e\n",
              "\u003c/table\u003e\n",
              "\u003c/div\u003e"
            ],
            "text/plain": [
              "                       sequence partition  ... is_viable     chip\n",
              "0  ADEEIRATNPIATEMYGSVSTNLQLGNR  designed  ...     False  harvard\n",
              "1  ADEEIRATNPVATEQYGSVSTNQQRQNR  designed  ...      True  harvard\n",
              "2  ADEEIRTTNPVATEQWGGVSTNLQIGNY  designed  ...      True  harvard\n",
              "3  ADEEIRTTNPVATEQYGEVSTNLQRGNR  designed  ...      True  harvard\n",
              "4  ADEEIRTTNPVATEQYGSVSTNLQRGNR  designed  ...      True  harvard\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "execution_count": 39,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "combined.head()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "last_runtime": {
        "build_target": "//research/biology/collaborations/aptamers:notebook",
        "kind": "shared"
      },
      "name": "data_prep.ipynb",
      "provenance": [
        {
          "file_id": "12Apk6W8VE6b1uJeDeE8zMDxIQoy7PmXb",
          "timestamp": 1593127225338
        },
        {
          "file_id": "1EQIf7CtSDR2_9XbbaVWWflGed3iv7_qp",
          "timestamp": 1593104311501
        },
        {
          "file_id": "1IzF054h5ytTwlJCu3jrM4m37IjpAfoq6",
          "timestamp": 1592708284079
        },
        {
          "file_id": "1PRds7BGpQ3VGGE3u69vIRK394IY0G08I",
          "timestamp": 1592703091207
        },
        {
          "file_id": "1M4_0I28I2vX17fhmjAgHOKWVtipblro-",
          "timestamp": 1592700744798
        },
        {
          "file_id": "1O4rVgAnAqZn8L91yriSwfucLYbUbsDut",
          "timestamp": 1592694353477
        },
        {
          "file_id": "1BMtY6RP5-wmWWnlUahAL6tNp_9Tsxv0f",
          "timestamp": 1592690804400
        }
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
